﻿import orange
import orngText
import orngMDS

##domain = orange.Domain([orange.EnumVariable(name="class_id", values=[1, -1]), orange.StringVariable("news_id")])
##
##
##foo = [orange.EnumVariable(name="class_id", values=["1", "-1"]), orange.StringVariable("news_id")]
##bar = [orange.StringVariable(str(i)) for i in range(10)]
##
##examples = []
##sd
##
##
##for n in  for news in db.query(News).filter(News.content != ''):
##    ex = orange.Example(domain, [news.content.encode("utf-8"), ["top", "new"][news.rank == None]])
##    examples.append(ex)
###examples = [orange.Example(domain, [news.content.encode("utf-8"), ["top", "new"][news.rank == None]]) for news in db.query(News).filter(News.content != '')]
##data = orange.ExampleTable(examples)

# ==== PRINT ====
for i in examples:
    print i["type"], i["text"].value[:50]

# process text, obtain TFIDF vectors
p = orngText.Preprocess(language="sl")
data = p.removeStopwordsFromExampleTable(data, 0)
data = p.lemmatizeExampleTable(data, 0)
data = orngText.bagOfWords(data)
bof = data
data = orngText.PreprocessorConstructor_tfidf()(data)(data)
data = orngText.Preprocessor_norm()(data, 2)

# compute distance as 1/cos(fi)
distance = orngText.cos(data, distance = 1)

# use MDS for visualization
print "running MDS"
mds=orngMDS.MDS(distance)
mds.run(100)

from pylab import *
colors = ["red", "yellow", "blue"]

points = []
for (i,d) in enumerate(data):
   points.append((mds.points[i][0], mds.points[i][1], d.getclass()))
for c in range(len(data.domain.classVar.values)):
   sel = filter(lambda x: x[-1]==c, points)
   x = [s[0] for s in sel if s[0] > -200000]
   y = [s[1] for s in sel if s[0] > -200000]
   scatter(x, y, c=colors[c])
savefig('mds-books.png', dpi=360)
